#ifndef __PAGR_LIB_PREPROCESSOR_H__
#define __PAGR_LIB_PREPROCESSOR_H__
#include"SplitToolJieba.h"
#include<vector>
#include<unordered_map>
#include<set>
using std::set;

using std::vector;
using std::unordered_map;
using std::pair;

class PageLibPreprocessor
{
public:

    PageLibPreprocessor(SplitTool* pt);

    void cutRedundantPage();

    void buildinvertindexMap();

    void storeOnDisk();

private:

    map<int,pair<int,int>> _offsetlib;
    unordered_map<string,vector<pair<int,double>>> _invertindexlib;
    SplitTool* _wordCutter;
};

#endif
